#!/usr/bin/env python
'''
简单的网页爬虫
依赖的库retrying、BeautifulSoup、colorama 自行百度安装
pip install retrying
pip install colorama
pip install beautifulsoup4
'''
from MyCrawler import Crawler
from TargetWebsite import TargetWebsite


class Main(object):

    def __init__(self):
        website1 = TargetWebsite("中国贸易金融网-金融资讯", "http://www.sinotf.com/GB/News/1002/", "utf-8",
                                 "div.list24px > ul > li > a", "div#endMain > div#endText")
        website2 = TargetWebsite("融站网-理财技巧", "http://www.erongzhan.com/jinrongzixun/licai/", "gbk",
                                 "div.deanarticername > a", "td#article_content")
        website3 = TargetWebsite("融站网-财经资讯", "http://www.erongzhan.com/jinrongzixun/caijing/", "gbk",
                                 "div.deanarticername > a", "td#article_content")
        website4 = TargetWebsite("融站网-互联网金融", "http://www.erongzhan.com/jinrongzixun/hujin/", "gbk",
                                 "div.deanarticername > a", "td#article_content")
        website5 = TargetWebsite("融站网-网贷观察", "http://www.erongzhan.com/jinrongzixun/wangdai/", "gbk",
                                 "div.deanarticername > a", "td#article_content")
        website6 = TargetWebsite("豫商贷-理财资讯", "https://www.zzwms.com/news/mtbd", "utf-8",
                                 "div.news_list > ul > li > a", "div.news_list_xq")

        self.crawler = Crawler()
        self.crawler.addTargetWebsite(website1).addTargetWebsite(website2).addTargetWebsite(
            website3).addTargetWebsite(website4).addTargetWebsite(website5).addTargetWebsite(website6)

    def run(self):
        self.crawler.clearOldSavedContent()
        self.crawler.start()

    def test(self):
        # 测试代码
        self.testCrawler = Crawler()
        self.testCrawler.clearOldSavedContent()
        self.testWebsite = TargetWebsite("融站网-网贷观察", "http://www.erongzhan.com/jinrongzixun/wangdai/",
                                         "gbk", "div.deanarticec > div.deanarticer > div.deanarticername > a", "td#article_content")

        self.testCrawler.testParseList(self.testWebsite)


if __name__ == '__main__':
    Main().run()
